﻿Imports HtmlAgilityPack
Imports System.Text.RegularExpressions
Imports System.Data.SQLite
Imports System.Text

Namespace OfflineLibrary_WPF
	Module PageSearch

		Private Class SearchResultsPage
			Dim sb As New StringBuilder

			Public Sub New()
				sb.AppendLine("<!DOCTYPE html><html><head><meta http-equiv=""Content-Type"" content=""text/html; charset=UTF-8"" /></head><body>")
			End Sub

			Public Sub AddResult(Title As String, Snippet As String, URL As String)
				AddResult(String.Format("<a href='{0}'>{1}<br />{2}</a>", URL, Title, Snippet))
			End Sub

			Public Sub AddResult(Title As String, Snippet As String, URL As Uri)
				AddResult(Title, Snippet, URL.AbsoluteUri)
			End Sub

			Public Sub AddResult(Result As String)
				sb.AppendLine("<p>")
				sb.AppendLine(Result)
				sb.AppendLine("</p>")
			End Sub

			Public Sub AddURLResult(Result As Uri)
				sb.AppendLine("<p>")
				sb.AppendLine(String.Format("<a href='{0}'>{0}</a>", Result))
				sb.AppendLine("</p>")
			End Sub

			Public Overrides Function ToString() As String
				Return sb.ToString & "</body></html>"
			End Function
		End Class

		Public Function GetSearchTermsFromPage(htmlPage As HtmlDocument) As String
			Return htmlPage.DocumentNode.SelectSingleNode("//div[@id='content']").InnerText.Trim
		End Function

		Public Function GetTitleFromPage(htmlPage As HtmlDocument) As String
			Return htmlPage.DocumentNode.SelectSingleNode("/html/head/title").InnerText
		End Function

		Public Function GetPagesWithSearchTerms(SearchText As String) As String

			Dim results As New SearchResultsPage

			Using conn As New SQLiteConnection(OfflineDataConnectionString)
				conn.Open()

				Dim sb As New StringBuilder
				sb.AppendLine("SELECT '<a href=''' || URL || '''>' || PageTitle || '<br />' || snippet(PageSearchData) || '</a>' AS PageSearchResults FROM PageSearchData JOIN ( ")
				sb.AppendLine("    SELECT rank(matchinfo(PageSearchData, 'pcnalsx')) AS rank")
				sb.AppendLine("    FROM PageSearchData")
				sb.AppendLine("    WHERE PageSearchData MATCH @SearchText")
				sb.AppendLine("    ORDER BY rank DESC ")
				sb.AppendLine("    LIMIT 100 OFFSET 0")
				sb.AppendLine(") AS ranktable")
				sb.AppendLine("WHERE PageSearchData MATCH @SearchText")
				sb.AppendLine("ORDER BY ranktable.rank DESC")

				Dim sql As String = sb.ToString

				Using cmd As New SQLiteCommand(sql, conn)

					cmd.Parameters.AddWithValue("@SearchText", SearchText)

					Using reader As SQLiteDataReader = cmd.ExecuteReader

						Do While reader.Read
							results.AddResult(reader.GetString(0))
						Loop

					End Using

				End Using

			End Using

			Return results.ToString
		End Function

		Public Function GetAlternateLanguagePages(URL As Uri, Optional DestinationLanguageCode As String = "") As String
			Dim URLPageSegment As String = Regex.Match(URL.AbsolutePath, "(/[0-9]+)+").Value
			Dim CurrentLanguageCode As String = Regex.Match(URL.AbsolutePath, "^/.+?/").Value

			Dim results As New SearchResultsPage

			Using conn As New SQLiteConnection(OfflineDataConnectionString)
				conn.Open()

				'for some reason the parameter version doesn't work

				'Dim sql As String = "SELECT URL FROM PageSearchData WHERE URL MATCH '@DestinationLanguageCode ""@URL"" -""@CurrentLanguage""'"
				'the quotes must only be there if destinationlangaugecode isn't empty
				Dim sql As String = String.Format("SELECT URL FROM PageSearchData WHERE URL MATCH '{0} ""{1}""'", If(String.IsNullOrWhiteSpace(DestinationLanguageCode), "", String.Format("""{0}""", DestinationLanguageCode)), URLPageSegment)
				Using cmd As New SQLiteCommand(sql, conn)

					'cmd.Parameters.AddWithValue("@DestinationLangaugeCode", DestinationLanguageCode)
					'cmd.Parameters.AddWithValue("@URL", URLPageSegment)
					'cmd.Parameters.AddWithValue("@CurrentLanguage", CurrentLanguageCode)

					Using reader As SQLiteDataReader = cmd.ExecuteReader

						Do While reader.Read
							Dim readURL As New Uri(reader.GetString(reader.GetOrdinal("URL")))
							If readURL.Segments.Count = URL.Segments.Count AndAlso (Not readURL.AbsolutePath.Contains(CurrentLanguageCode) OrElse readURL.AbsolutePath.Contains(DestinationLanguageCode)) Then
								results.AddURLResult(readURL)
							End If
						Loop

					End Using

				End Using

			End Using

			Return results.ToString

		End Function

		Public Sub TestReadRankData()
			'select matchinfo(pagesearchdata , 'pcsx') from pagesearchdata where url match 'zh-Hans OR wol OR b OR r23 OR lp-chs OR 1 OR 1'

			Dim myMatchInfo As New List(Of MatchInfo)
			Dim myMatchList As New List(Of KeyValuePair(Of String, String))

			Using conn As New SQLiteConnection(OfflineDataConnectionString)
				conn.Open()

				'Dim sql As String = "select matchinfo(pagesearchdata , 'pcnalsx') as matchinfo, url from pagesearchdata where url match '""/2/1"" -en'"
				Dim sql As String = "select rank(matchinfo(pagesearchdata , 'pcnalsx')) as matchinfo, url from pagesearchdata where url match '/2/1'"
				Using cmd As New SQLiteCommand(sql, conn)

					'cmd.Parameters.AddWithValue("@SearchText", SearchText)

					Using reader As SQLiteDataReader = cmd.ExecuteReader

						Do While reader.Read
							'myMatchInfo.Add(New MatchInfo(reader.GetString(reader.GetOrdinal("URL")), GetBytes(reader.GetOrdinal("matchinfo"), reader)))
							myMatchList.Add(New KeyValuePair(Of String, String)(reader.GetString(reader.GetOrdinal("URL")), reader.GetFloat(reader.GetOrdinal("matchinfo"))))
						Loop

					End Using

				End Using

			End Using

			'Dim test As Integer = myMatchInfo(9).LongestPhraseMatch
			'Dim test2 As Integer = myMatchInfo(11).LongestPhraseMatch

			'Dim test3 = myMatchInfo.Where(Function(x) x.LongestPhraseMatch >= 2)

		End Sub

		Private Function GetBytes(columnOrdinal As Integer, reader As SQLiteDataReader) As Byte()
			Const CHUNK_SIZE As Integer = 2 * 1024
			Dim buffer As Byte() = New Byte(CHUNK_SIZE - 1) {}
			Dim bytesRead As Long
			Dim fieldOffset As Long = 0
			Using stream As New IO.MemoryStream()
				While (InlineAssignHelper(bytesRead, reader.GetBytes(columnOrdinal, fieldOffset, buffer, 0, buffer.Length))) > 0
					stream.Write(buffer, 0, CInt(bytesRead))
					fieldOffset += bytesRead
				End While
				Return stream.ToArray()
			End Using
		End Function

		Private Function InlineAssignHelper(Of T)(ByRef target As T, value As T) As T
			target = value
			Return value
		End Function

	End Module

	Public Class MatchInfo
		Property URL As Uri
		Property matchablePhrases As Integer
		Property userDefinedColumns As Integer
		Property totalDocuments As Integer
		Private _int32HitData As List(Of Integer)
		Private _longestSubsequencePhraseMatches As New List(Of Integer)
		Private _tokensInDocument As New List(Of Integer)
		Private _averageTokensInDocument As New List(Of Integer)

#Region "Properties"
		Private _max_hits_this_row As Integer?
		Public ReadOnly Property max_hits_this_row As Integer
			Get
				If _max_hits_this_row Is Nothing Then
					_max_hits_this_row = 0
					For p = 0 To matchablePhrases - 1
						For c = 0 To userDefinedColumns - 1
							Dim myHitsThisRow As Integer = hits_this_row(p, c)
							If myHitsThisRow > _max_hits_this_row Then
								_max_hits_this_row = myHitsThisRow
							End If
						Next
					Next
				End If

				Return _max_hits_this_row
			End Get
		End Property

		Private _max_hits_all_rows As Integer?
		Public ReadOnly Property max_hits_all_rows As Integer
			Get
				If _max_hits_all_rows Is Nothing Then
					_max_hits_all_rows = 0
					For p = 0 To matchablePhrases - 1
						For c = 0 To userDefinedColumns - 1
							Dim myHitsAllRows As Integer = hits_all_rows(p, c)
							If myHitsAllRows > _max_hits_all_rows Then
								_max_hits_all_rows = myHitsAllRows
							End If
						Next
					Next
				End If

				Return _max_hits_all_rows
			End Get
		End Property

		Private _max_docs_with_hits As Integer?
		Public ReadOnly Property max_docs_with_hits As Integer
			Get
				If _max_docs_with_hits Is Nothing Then
					_max_docs_with_hits = 0
					For p = 0 To matchablePhrases - 1
						For c = 0 To userDefinedColumns - 1
							Dim myDocsWithHits As Integer = docs_with_hits(p, c)
							If myDocsWithHits > _max_docs_with_hits Then
								_max_docs_with_hits = myDocsWithHits
							End If
						Next
					Next
				End If

				Return _max_docs_with_hits
			End Get
		End Property

		Private _BM25Rank As Double?
		Public ReadOnly Property BM25Rank As Double
			Get
				If _BM25Rank Is Nothing Then
					_BM25Rank = 0
					'calculate BM25 Rank
					'http://en.wikipedia.org/wiki/Okapi_BM25

					'k1, calibrates the document term frequency scaling. Having k1 as 0 corresponds to a binary model – no term frequency. Increasing k1 will give rare words more boost.
					'b, calibrates the scaling by document length, and can take values from 0 to 1, where having 0 means no length normalization and having 1 corresponds to fully scaling the term weight by the document length.

					Dim k1 As Double = 1.2
					Dim b As Double = 0.75

					For column = 0 To userDefinedColumns - 1
						For phrase = 0 To matchablePhrases - 1
							Dim IDF As Double = Math.Log((totalDocuments - hits_all_rows(phrase, column) + 0.5) / (hits_all_rows(phrase, column) + 0.5))
							Dim score As Double = (IDF * ((hits_this_row(phrase, column) * (k1 + 1)) / (hits_this_row(phrase, column) + k1 * (1 - b + b * _tokensInDocument(column) / _averageTokensInDocument(column)))))

							'ignore very common words
							If score < 0 Then
								score = 0
							End If
							_BM25Rank += score
						Next
					Next

				End If

				Return _BM25Rank
			End Get
		End Property
#End Region

		Public Sub New(URL As String, raw_pcnalsx_MatchInfo As Byte())
			Try
				Me.URL = New Uri(URL)
			Catch ex As Exception
			End Try

			Dim int32_pcsx_MatchInfo As New List(Of Integer)
			For i = 0 To raw_pcnalsx_MatchInfo.Length - 1 Step 4
				int32_pcsx_MatchInfo.Add(BitConverter.ToUInt32(raw_pcnalsx_MatchInfo, i))
			Next

			'take the raw data and parse it out
			Me.matchablePhrases = int32_pcsx_MatchInfo(0)
			int32_pcsx_MatchInfo.RemoveAt(0)

			Me.userDefinedColumns = int32_pcsx_MatchInfo(0)
			int32_pcsx_MatchInfo.RemoveAt(0)

			Me.totalDocuments = int32_pcsx_MatchInfo(0)
			int32_pcsx_MatchInfo.RemoveAt(0)

			'remember that the columns are 0-based
			For i = 0 To userDefinedColumns - 1
				_averageTokensInDocument.Add(int32_pcsx_MatchInfo(0))
				int32_pcsx_MatchInfo.RemoveAt(0)
			Next

			For i = 0 To userDefinedColumns - 1
				_tokensInDocument.Add(int32_pcsx_MatchInfo(0))
				int32_pcsx_MatchInfo.RemoveAt(0)
			Next

			For i = 0 To userDefinedColumns - 1
				_longestSubsequencePhraseMatches.Add(int32_pcsx_MatchInfo(0))
				int32_pcsx_MatchInfo.RemoveAt(0)
			Next

			_int32HitData = New List(Of Integer)(int32_pcsx_MatchInfo)

		End Sub

		Public Function hits_this_row(phrase As Integer, column As Integer) As Integer
			Return _int32HitData(3 * (column + phrase * userDefinedColumns) + 0)
		End Function

		Public Function hits_all_rows(phrase As Integer, column As Integer) As Integer
			Return _int32HitData(3 * (column + phrase * userDefinedColumns) + 1)
		End Function

		Public Function docs_with_hits(phrase As Integer, column As Integer) As Integer
			Return _int32HitData(3 * (column + phrase * userDefinedColumns) + 2)
		End Function
	End Class

	<SQLiteFunction("Rank", 1, FunctionType.Scalar)>
	Public Class Rank
		Inherits SQLiteFunction

		Public Overrides Function Invoke(args() As Object) As Object
			Return New MatchInfo(Nothing, args(0)).BM25Rank
		End Function

	End Class
End Namespace